sidekiq 6.1.1 → 6.5.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Changes.md +250 -3
- data/LICENSE +3 -3
- data/README.md +10 -6
- data/bin/sidekiq +3 -3
- data/bin/sidekiqload +70 -66
- data/bin/sidekiqmon +1 -1
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +352 -156
- data/lib/sidekiq/cli.rb +86 -41
- data/lib/sidekiq/client.rb +49 -73
- data/lib/sidekiq/{util.rb → component.rb} +12 -14
- data/lib/sidekiq/delay.rb +3 -1
- data/lib/sidekiq/extensions/action_mailer.rb +3 -2
- data/lib/sidekiq/extensions/active_record.rb +1 -1
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
- data/lib/sidekiq/fetch.rb +31 -20
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +16 -28
- data/lib/sidekiq/job_retry.rb +79 -59
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +126 -65
- data/lib/sidekiq/logger.rb +11 -20
- data/lib/sidekiq/manager.rb +35 -34
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +88 -42
- data/lib/sidekiq/middleware/current_attributes.rb +63 -0
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +2 -2
- data/lib/sidekiq/paginator.rb +17 -9
- data/lib/sidekiq/processor.rb +47 -41
- data/lib/sidekiq/rails.rb +32 -4
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +84 -55
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +96 -32
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +38 -39
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +41 -16
- data/lib/sidekiq/web/csrf_protection.rb +32 -5
- data/lib/sidekiq/web/helpers.rb +52 -30
- data/lib/sidekiq/web/router.rb +4 -1
- data/lib/sidekiq/web.rb +38 -78
- data/lib/sidekiq/worker.rb +142 -16
- data/lib/sidekiq.rb +114 -31
- data/sidekiq.gemspec +12 -4
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +114 -60
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +50 -67
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application-dark.css +61 -51
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +84 -243
- data/web/locales/ar.yml +8 -2
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +11 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +8 -1
- data/web/locales/ja.yml +10 -0
- data/web/locales/lt.yml +1 -1
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/locales/zh-cn.yml +36 -11
- data/web/locales/zh-tw.yml +32 -7
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +1 -1
- data/web/views/_nav.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +57 -21
- data/web/views/dashboard.erb +23 -14
- data/web/views/dead.erb +1 -1
- data/web/views/layout.erb +2 -1
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/morgue.erb +6 -6
- data/web/views/queue.erb +15 -11
- data/web/views/queues.erb +4 -4
- data/web/views/retries.erb +7 -7
- data/web/views/retry.erb +1 -1
- data/web/views/scheduled.erb +1 -1
- metadata +52 -39
- data/.circleci/config.yml +0 -71
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.standard.yml +0 -20
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/6.0-Upgrade.md +0 -72
- data/COMM-LICENSE +0 -97
- data/Ent-2.0-Upgrade.md +0 -37
- data/Ent-Changes.md +0 -275
- data/Gemfile +0 -24
- data/Gemfile.lock +0 -208
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-5.0-Upgrade.md +0 -25
- data/Pro-Changes.md +0 -795
- data/Rakefile +0 -10
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
- data/lib/sidekiq/exception_handler.rb +0 -27
data/lib/sidekiq/cli.rb
CHANGED
@@ -9,18 +9,34 @@ require "erb"
|
|
9
9
|
require "fileutils"
|
10
10
|
|
11
11
|
require "sidekiq"
|
12
|
+
require "sidekiq/component"
|
12
13
|
require "sidekiq/launcher"
|
13
|
-
require "sidekiq/util"
|
14
14
|
|
15
|
-
module
|
15
|
+
# module ScoutApm
|
16
|
+
# VERSION = "5.3.1"
|
17
|
+
# end
|
18
|
+
fail <<~EOM if defined?(ScoutApm::VERSION) && ScoutApm::VERSION < "5.2.0"
|
19
|
+
|
20
|
+
|
21
|
+
scout_apm v#{ScoutApm::VERSION} is unsafe with Sidekiq 6.5. Please run `bundle up scout_apm` to upgrade to 5.2.0 or greater.
|
22
|
+
|
23
|
+
|
24
|
+
EOM
|
25
|
+
|
26
|
+
module Sidekiq # :nodoc:
|
16
27
|
class CLI
|
17
|
-
include
|
28
|
+
include Sidekiq::Component
|
18
29
|
include Singleton unless $TESTING
|
19
30
|
|
20
31
|
attr_accessor :launcher
|
21
32
|
attr_accessor :environment
|
33
|
+
attr_accessor :config
|
34
|
+
|
35
|
+
def parse(args = ARGV.dup)
|
36
|
+
@config = Sidekiq
|
37
|
+
@config[:error_handlers].clear
|
38
|
+
@config[:error_handlers] << @config.method(:default_error_handler)
|
22
39
|
|
23
|
-
def parse(args = ARGV)
|
24
40
|
setup_options(args)
|
25
41
|
initialize_logger
|
26
42
|
validate!
|
@@ -36,7 +52,7 @@ module Sidekiq
|
|
36
52
|
def run(boot_app: true)
|
37
53
|
boot_application if boot_app
|
38
54
|
|
39
|
-
if environment == "development" && $stdout.tty? &&
|
55
|
+
if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
40
56
|
print_banner
|
41
57
|
end
|
42
58
|
logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
|
@@ -44,9 +60,17 @@ module Sidekiq
|
|
44
60
|
self_read, self_write = IO.pipe
|
45
61
|
sigs = %w[INT TERM TTIN TSTP]
|
46
62
|
# USR1 and USR2 don't work on the JVM
|
47
|
-
sigs << "USR2"
|
63
|
+
sigs << "USR2" if Sidekiq.pro? && !jruby?
|
48
64
|
sigs.each do |sig|
|
49
|
-
trap
|
65
|
+
old_handler = Signal.trap(sig) do
|
66
|
+
if old_handler.respond_to?(:call)
|
67
|
+
begin
|
68
|
+
old_handler.call
|
69
|
+
rescue Exception => exc
|
70
|
+
# signal handlers can't use Logger so puts only
|
71
|
+
puts ["Error in #{sig} handler", exc].inspect
|
72
|
+
end
|
73
|
+
end
|
50
74
|
self_write.puts(sig)
|
51
75
|
end
|
52
76
|
rescue ArgumentError
|
@@ -59,27 +83,40 @@ module Sidekiq
|
|
59
83
|
|
60
84
|
# touch the connection pool so it is created before we
|
61
85
|
# fire startup and start multithreading.
|
62
|
-
|
86
|
+
info = @config.redis_info
|
87
|
+
ver = info["redis_version"]
|
63
88
|
raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
|
64
89
|
|
90
|
+
maxmemory_policy = info["maxmemory_policy"]
|
91
|
+
if maxmemory_policy != "noeviction"
|
92
|
+
logger.warn <<~EOM
|
93
|
+
|
94
|
+
|
95
|
+
WARNING: Your Redis instance will evict Sidekiq data under heavy load.
|
96
|
+
The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
|
97
|
+
See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
|
98
|
+
|
99
|
+
EOM
|
100
|
+
end
|
101
|
+
|
65
102
|
# Since the user can pass us a connection pool explicitly in the initializer, we
|
66
103
|
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
67
|
-
cursize =
|
68
|
-
needed =
|
104
|
+
cursize = @config.redis_pool.size
|
105
|
+
needed = @config[:concurrency] + 2
|
69
106
|
raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
|
70
107
|
|
71
108
|
# cache process identity
|
72
|
-
|
109
|
+
@config[:identity] = identity
|
73
110
|
|
74
111
|
# Touch middleware so it isn't lazy loaded by multiple threads, #3043
|
75
|
-
|
112
|
+
@config.server_middleware
|
76
113
|
|
77
114
|
# Before this point, the process is initializing with just the main thread.
|
78
115
|
# Starting here the process will now have multiple threads running.
|
79
116
|
fire_event(:startup, reverse: false, reraise: true)
|
80
117
|
|
81
|
-
logger.debug { "Client Middleware: #{
|
82
|
-
logger.debug { "Server Middleware: #{
|
118
|
+
logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
|
119
|
+
logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
|
83
120
|
|
84
121
|
launch(self_read)
|
85
122
|
end
|
@@ -89,13 +126,13 @@ module Sidekiq
|
|
89
126
|
logger.info "Starting processing, hit Ctrl-C to stop"
|
90
127
|
end
|
91
128
|
|
92
|
-
@launcher = Sidekiq::Launcher.new(
|
129
|
+
@launcher = Sidekiq::Launcher.new(@config)
|
93
130
|
|
94
131
|
begin
|
95
132
|
launcher.run
|
96
133
|
|
97
|
-
while
|
98
|
-
signal =
|
134
|
+
while self_read.wait_readable
|
135
|
+
signal = self_read.gets.strip
|
99
136
|
handle_signal(signal)
|
100
137
|
end
|
101
138
|
rescue Interrupt
|
@@ -152,25 +189,25 @@ module Sidekiq
|
|
152
189
|
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
153
190
|
"TERM" => ->(cli) { raise Interrupt },
|
154
191
|
"TSTP" => ->(cli) {
|
155
|
-
|
192
|
+
cli.logger.info "Received TSTP, no longer accepting new work"
|
156
193
|
cli.launcher.quiet
|
157
194
|
},
|
158
195
|
"TTIN" => ->(cli) {
|
159
196
|
Thread.list.each do |thread|
|
160
|
-
|
197
|
+
cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
|
161
198
|
if thread.backtrace
|
162
|
-
|
199
|
+
cli.logger.warn thread.backtrace.join("\n")
|
163
200
|
else
|
164
|
-
|
201
|
+
cli.logger.warn "<no backtrace available>"
|
165
202
|
end
|
166
203
|
end
|
167
204
|
}
|
168
205
|
}
|
169
|
-
UNHANDLED_SIGNAL_HANDLER = ->(cli) {
|
206
|
+
UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
|
170
207
|
SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
|
171
208
|
|
172
209
|
def handle_signal(sig)
|
173
|
-
|
210
|
+
logger.debug "Got #{sig} signal"
|
174
211
|
SIGNAL_HANDLERS[sig].call(self)
|
175
212
|
end
|
176
213
|
|
@@ -188,6 +225,7 @@ module Sidekiq
|
|
188
225
|
# Both Sinatra 2.0+ and Sidekiq support this term.
|
189
226
|
# RAILS_ENV and RACK_ENV are there for legacy support.
|
190
227
|
@environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
228
|
+
config[:environment] = @environment
|
191
229
|
end
|
192
230
|
|
193
231
|
def symbolize_keys_deep!(hash)
|
@@ -216,7 +254,7 @@ module Sidekiq
|
|
216
254
|
config_dir = if File.directory?(opts[:require].to_s)
|
217
255
|
File.join(opts[:require], "config")
|
218
256
|
else
|
219
|
-
File.join(
|
257
|
+
File.join(@config[:require], "config")
|
220
258
|
end
|
221
259
|
|
222
260
|
%w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
|
@@ -233,27 +271,23 @@ module Sidekiq
|
|
233
271
|
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
234
272
|
|
235
273
|
# merge with defaults
|
236
|
-
|
237
|
-
end
|
238
|
-
|
239
|
-
def options
|
240
|
-
Sidekiq.options
|
274
|
+
@config.merge!(opts)
|
241
275
|
end
|
242
276
|
|
243
277
|
def boot_application
|
244
278
|
ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
|
245
279
|
|
246
|
-
if File.directory?(
|
280
|
+
if File.directory?(@config[:require])
|
247
281
|
require "rails"
|
248
282
|
if ::Rails::VERSION::MAJOR < 5
|
249
283
|
raise "Sidekiq no longer supports this version of Rails"
|
250
284
|
else
|
251
285
|
require "sidekiq/rails"
|
252
|
-
require File.expand_path("#{
|
286
|
+
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
253
287
|
end
|
254
|
-
|
288
|
+
@config[:tag] ||= default_tag
|
255
289
|
else
|
256
|
-
require
|
290
|
+
require @config[:require]
|
257
291
|
end
|
258
292
|
end
|
259
293
|
|
@@ -270,18 +304,18 @@ module Sidekiq
|
|
270
304
|
end
|
271
305
|
|
272
306
|
def validate!
|
273
|
-
if !File.exist?(
|
274
|
-
(File.directory?(
|
307
|
+
if !File.exist?(@config[:require]) ||
|
308
|
+
(File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
|
275
309
|
logger.info "=================================================================="
|
276
310
|
logger.info " Please point Sidekiq to a Rails application or a Ruby file "
|
277
|
-
logger.info " to load your
|
311
|
+
logger.info " to load your job classes with -r [DIR|FILE]."
|
278
312
|
logger.info "=================================================================="
|
279
313
|
logger.info @parser
|
280
314
|
die(1)
|
281
315
|
end
|
282
316
|
|
283
317
|
[:concurrency, :timeout].each do |opt|
|
284
|
-
raise ArgumentError, "#{opt}: #{
|
318
|
+
raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
|
285
319
|
end
|
286
320
|
end
|
287
321
|
|
@@ -315,7 +349,7 @@ module Sidekiq
|
|
315
349
|
parse_queue opts, queue, weight
|
316
350
|
end
|
317
351
|
|
318
|
-
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with
|
352
|
+
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
|
319
353
|
opts[:require] = arg
|
320
354
|
end
|
321
355
|
|
@@ -355,11 +389,13 @@ module Sidekiq
|
|
355
389
|
end
|
356
390
|
|
357
391
|
def initialize_logger
|
358
|
-
|
392
|
+
@config.logger.level = ::Logger::DEBUG if @config[:verbose]
|
359
393
|
end
|
360
394
|
|
361
395
|
def parse_config(path)
|
362
|
-
|
396
|
+
erb = ERB.new(File.read(path))
|
397
|
+
erb.filename = File.expand_path(path)
|
398
|
+
opts = load_yaml(erb.result) || {}
|
363
399
|
|
364
400
|
if opts.respond_to? :deep_symbolize_keys!
|
365
401
|
opts.deep_symbolize_keys!
|
@@ -375,6 +411,14 @@ module Sidekiq
|
|
375
411
|
opts
|
376
412
|
end
|
377
413
|
|
414
|
+
def load_yaml(src)
|
415
|
+
if Psych::VERSION > "4.0"
|
416
|
+
YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
|
417
|
+
else
|
418
|
+
YAML.load(src)
|
419
|
+
end
|
420
|
+
end
|
421
|
+
|
378
422
|
def parse_queues(opts, queues_and_weights)
|
379
423
|
queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
|
380
424
|
end
|
@@ -383,7 +427,7 @@ module Sidekiq
|
|
383
427
|
opts[:queues] ||= []
|
384
428
|
opts[:strict] = true if opts[:strict].nil?
|
385
429
|
raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
|
386
|
-
[weight.to_i, 1].max.times { opts[:queues] << queue }
|
430
|
+
[weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
|
387
431
|
opts[:strict] = false if weight.to_i > 0
|
388
432
|
end
|
389
433
|
|
@@ -394,3 +438,4 @@ module Sidekiq
|
|
394
438
|
end
|
395
439
|
|
396
440
|
require "sidekiq/systemd"
|
441
|
+
require "sidekiq/metrics/tracking" if ENV["SIDEKIQ_METRICS_BETA"]
|
data/lib/sidekiq/client.rb
CHANGED
@@ -2,9 +2,12 @@
|
|
2
2
|
|
3
3
|
require "securerandom"
|
4
4
|
require "sidekiq/middleware/chain"
|
5
|
+
require "sidekiq/job_util"
|
5
6
|
|
6
7
|
module Sidekiq
|
7
8
|
class Client
|
9
|
+
include Sidekiq::JobUtil
|
10
|
+
|
8
11
|
##
|
9
12
|
# Define client-side middleware:
|
10
13
|
#
|
@@ -12,14 +15,14 @@ module Sidekiq
|
|
12
15
|
# client.middleware do |chain|
|
13
16
|
# chain.use MyClientMiddleware
|
14
17
|
# end
|
15
|
-
# client.push('class' => '
|
18
|
+
# client.push('class' => 'SomeJob', 'args' => [1,2,3])
|
16
19
|
#
|
17
20
|
# All client instances default to the globally-defined
|
18
21
|
# Sidekiq.client_middleware but you can change as necessary.
|
19
22
|
#
|
20
23
|
def middleware(&block)
|
21
24
|
@chain ||= Sidekiq.client_middleware
|
22
|
-
if
|
25
|
+
if block
|
23
26
|
@chain = @chain.dup
|
24
27
|
yield @chain
|
25
28
|
end
|
@@ -46,16 +49,16 @@ module Sidekiq
|
|
46
49
|
# The main method used to push a job to Redis. Accepts a number of options:
|
47
50
|
#
|
48
51
|
# queue - the named queue to use, default 'default'
|
49
|
-
# class - the
|
52
|
+
# class - the job class to call, required
|
50
53
|
# args - an array of simple arguments to the perform method, must be JSON-serializable
|
51
54
|
# at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
|
52
55
|
# retry - whether to retry this job if it fails, default true or an integer number of retries
|
53
56
|
# backtrace - whether to save any error backtrace, default false
|
54
57
|
#
|
55
58
|
# If class is set to the class name, the jobs' options will be based on Sidekiq's default
|
56
|
-
#
|
59
|
+
# job options. Otherwise, they will be based on the job class's options.
|
57
60
|
#
|
58
|
-
# Any options valid for a
|
61
|
+
# Any options valid for a job class's sidekiq_options are also available here.
|
59
62
|
#
|
60
63
|
# All options must be strings, not symbols. NB: because we are serializing to JSON, all
|
61
64
|
# symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
|
@@ -64,13 +67,15 @@ module Sidekiq
|
|
64
67
|
# Returns a unique Job ID. If middleware stops the job, nil will be returned instead.
|
65
68
|
#
|
66
69
|
# Example:
|
67
|
-
# push('queue' => 'my_queue', 'class' =>
|
70
|
+
# push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
|
68
71
|
#
|
69
72
|
def push(item)
|
70
73
|
normed = normalize_item(item)
|
71
|
-
payload =
|
72
|
-
|
74
|
+
payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
|
75
|
+
normed
|
76
|
+
end
|
73
77
|
if payload
|
78
|
+
verify_json(payload)
|
74
79
|
raw_push([payload])
|
75
80
|
payload["jid"]
|
76
81
|
end
|
@@ -95,15 +100,20 @@ module Sidekiq
|
|
95
100
|
return [] if args.empty? # no jobs to push
|
96
101
|
|
97
102
|
at = items.delete("at")
|
98
|
-
raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
|
103
|
+
raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
|
99
104
|
raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
|
100
105
|
|
106
|
+
jid = items.delete("jid")
|
107
|
+
raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
|
108
|
+
|
101
109
|
normed = normalize_item(items)
|
102
110
|
payloads = args.map.with_index { |job_args, index|
|
103
|
-
copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12)
|
111
|
+
copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
|
104
112
|
copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
|
105
|
-
|
106
|
-
|
113
|
+
result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
|
114
|
+
verify_json(copy)
|
115
|
+
copy
|
116
|
+
end
|
107
117
|
result || nil
|
108
118
|
}.compact
|
109
119
|
|
@@ -116,8 +126,8 @@ module Sidekiq
|
|
116
126
|
#
|
117
127
|
# pool = ConnectionPool.new { Redis.new }
|
118
128
|
# Sidekiq::Client.via(pool) do
|
119
|
-
#
|
120
|
-
#
|
129
|
+
# SomeJob.perform_async(1,2,3)
|
130
|
+
# SomeOtherJob.perform_async(1,2,3)
|
121
131
|
# end
|
122
132
|
#
|
123
133
|
# Generally this is only needed for very large Sidekiq installs processing
|
@@ -142,10 +152,10 @@ module Sidekiq
|
|
142
152
|
end
|
143
153
|
|
144
154
|
# Resque compatibility helpers. Note all helpers
|
145
|
-
# should go through
|
155
|
+
# should go through Sidekiq::Job#client_push.
|
146
156
|
#
|
147
157
|
# Example usage:
|
148
|
-
# Sidekiq::Client.enqueue(
|
158
|
+
# Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
|
149
159
|
#
|
150
160
|
# Messages are enqueued to the 'default' queue.
|
151
161
|
#
|
@@ -154,19 +164,19 @@ module Sidekiq
|
|
154
164
|
end
|
155
165
|
|
156
166
|
# Example usage:
|
157
|
-
# Sidekiq::Client.enqueue_to(:queue_name,
|
167
|
+
# Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
|
158
168
|
#
|
159
169
|
def enqueue_to(queue, klass, *args)
|
160
170
|
klass.client_push("queue" => queue, "class" => klass, "args" => args)
|
161
171
|
end
|
162
172
|
|
163
173
|
# Example usage:
|
164
|
-
# Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes,
|
174
|
+
# Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
|
165
175
|
#
|
166
176
|
def enqueue_to_in(queue, interval, klass, *args)
|
167
177
|
int = interval.to_f
|
168
178
|
now = Time.now.to_f
|
169
|
-
ts = (int < 1_000_000_000 ? now + int : int)
|
179
|
+
ts = ((int < 1_000_000_000) ? now + int : int)
|
170
180
|
|
171
181
|
item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
|
172
182
|
item.delete("at") if ts <= now
|
@@ -175,7 +185,7 @@ module Sidekiq
|
|
175
185
|
end
|
176
186
|
|
177
187
|
# Example usage:
|
178
|
-
# Sidekiq::Client.enqueue_in(3.minutes,
|
188
|
+
# Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
|
179
189
|
#
|
180
190
|
def enqueue_in(interval, klass, *args)
|
181
191
|
klass.perform_in(interval, *args)
|
@@ -186,8 +196,23 @@ module Sidekiq
|
|
186
196
|
|
187
197
|
def raw_push(payloads)
|
188
198
|
@redis_pool.with do |conn|
|
189
|
-
|
190
|
-
|
199
|
+
retryable = true
|
200
|
+
begin
|
201
|
+
conn.pipelined do |pipeline|
|
202
|
+
atomic_push(pipeline, payloads)
|
203
|
+
end
|
204
|
+
rescue RedisConnection.adapter::BaseError => ex
|
205
|
+
# 2550 Failover can cause the server to become a replica, need
|
206
|
+
# to disconnect and reopen the socket to get back to the primary.
|
207
|
+
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
208
|
+
# 4985 Use the same logic when a blocking command is force-unblocked
|
209
|
+
# The retry logic is copied from sidekiq.rb
|
210
|
+
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
211
|
+
conn.disconnect!
|
212
|
+
retryable = false
|
213
|
+
retry
|
214
|
+
end
|
215
|
+
raise
|
191
216
|
end
|
192
217
|
end
|
193
218
|
true
|
@@ -195,7 +220,7 @@ module Sidekiq
|
|
195
220
|
|
196
221
|
def atomic_push(conn, payloads)
|
197
222
|
if payloads.first.key?("at")
|
198
|
-
conn.zadd("schedule", payloads.
|
223
|
+
conn.zadd("schedule", payloads.flat_map { |hash|
|
199
224
|
at = hash.delete("at").to_s
|
200
225
|
[at, Sidekiq.dump_json(hash)]
|
201
226
|
})
|
@@ -206,58 +231,9 @@ module Sidekiq
|
|
206
231
|
entry["enqueued_at"] = now
|
207
232
|
Sidekiq.dump_json(entry)
|
208
233
|
}
|
209
|
-
conn.sadd("queues", queue)
|
234
|
+
conn.sadd("queues", [queue])
|
210
235
|
conn.lpush("queue:#{queue}", to_push)
|
211
236
|
end
|
212
237
|
end
|
213
|
-
|
214
|
-
def process_single(worker_class, item)
|
215
|
-
queue = item["queue"]
|
216
|
-
|
217
|
-
middleware.invoke(worker_class, item, queue, @redis_pool) do
|
218
|
-
item
|
219
|
-
end
|
220
|
-
end
|
221
|
-
|
222
|
-
def validate(item)
|
223
|
-
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
|
224
|
-
raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
|
225
|
-
raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
226
|
-
raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
|
227
|
-
raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
|
228
|
-
end
|
229
|
-
|
230
|
-
def normalize_item(item)
|
231
|
-
# 6.0.0 push_bulk bug, #4321
|
232
|
-
# TODO Remove after a while...
|
233
|
-
item.delete("at") if item.key?("at") && item["at"].nil?
|
234
|
-
|
235
|
-
validate(item)
|
236
|
-
# raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
|
237
|
-
|
238
|
-
# merge in the default sidekiq_options for the item's class and/or wrapped element
|
239
|
-
# this allows ActiveJobs to control sidekiq_options too.
|
240
|
-
defaults = normalized_hash(item["class"])
|
241
|
-
defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
|
242
|
-
item = defaults.merge(item)
|
243
|
-
|
244
|
-
raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
|
245
|
-
|
246
|
-
item["class"] = item["class"].to_s
|
247
|
-
item["queue"] = item["queue"].to_s
|
248
|
-
item["jid"] ||= SecureRandom.hex(12)
|
249
|
-
item["created_at"] ||= Time.now.to_f
|
250
|
-
|
251
|
-
item
|
252
|
-
end
|
253
|
-
|
254
|
-
def normalized_hash(item_class)
|
255
|
-
if item_class.is_a?(Class)
|
256
|
-
raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
|
257
|
-
item_class.get_sidekiq_options
|
258
|
-
else
|
259
|
-
Sidekiq.default_worker_options
|
260
|
-
end
|
261
|
-
end
|
262
238
|
end
|
263
239
|
end
|
@@ -1,15 +1,8 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "socket"
|
4
|
-
require "securerandom"
|
5
|
-
require "sidekiq/exception_handler"
|
6
|
-
|
7
1
|
module Sidekiq
|
8
2
|
##
|
9
|
-
#
|
10
|
-
#
|
11
|
-
|
12
|
-
include ExceptionHandler
|
3
|
+
# Sidekiq::Component assumes a config instance is available at @config
|
4
|
+
module Component # :nodoc:
|
5
|
+
attr_reader :config
|
13
6
|
|
14
7
|
def watchdog(last_words)
|
15
8
|
yield
|
@@ -26,11 +19,11 @@ module Sidekiq
|
|
26
19
|
end
|
27
20
|
|
28
21
|
def logger
|
29
|
-
|
22
|
+
config.logger
|
30
23
|
end
|
31
24
|
|
32
25
|
def redis(&block)
|
33
|
-
|
26
|
+
config.redis(&block)
|
34
27
|
end
|
35
28
|
|
36
29
|
def tid
|
@@ -49,11 +42,16 @@ module Sidekiq
|
|
49
42
|
@@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
|
50
43
|
end
|
51
44
|
|
45
|
+
def handle_exception(ex, ctx = {})
|
46
|
+
config.handle_exception(ex, ctx)
|
47
|
+
end
|
48
|
+
|
52
49
|
def fire_event(event, options = {})
|
50
|
+
oneshot = options.fetch(:oneshot, true)
|
53
51
|
reverse = options[:reverse]
|
54
52
|
reraise = options[:reraise]
|
55
53
|
|
56
|
-
arr =
|
54
|
+
arr = config[:lifecycle_events][event]
|
57
55
|
arr.reverse! if reverse
|
58
56
|
arr.each do |block|
|
59
57
|
block.call
|
@@ -61,7 +59,7 @@ module Sidekiq
|
|
61
59
|
handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
|
62
60
|
raise ex if reraise
|
63
61
|
end
|
64
|
-
arr.clear
|
62
|
+
arr.clear if oneshot # once we've fired an event, we never fire it again
|
65
63
|
end
|
66
64
|
end
|
67
65
|
end
|
data/lib/sidekiq/delay.rb
CHANGED
@@ -1,8 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module Sidekiq
|
3
|
+
module Sidekiq # :nodoc:
|
4
4
|
module Extensions
|
5
5
|
def self.enable_delay!
|
6
|
+
warn "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0", uplevel: 1
|
7
|
+
|
6
8
|
if defined?(::ActiveSupport)
|
7
9
|
require "sidekiq/extensions/active_record"
|
8
10
|
require "sidekiq/extensions/action_mailer"
|
@@ -5,9 +5,10 @@ require "sidekiq/extensions/generic_proxy"
|
|
5
5
|
module Sidekiq
|
6
6
|
module Extensions
|
7
7
|
##
|
8
|
-
# Adds
|
9
|
-
# delivery to Sidekiq.
|
8
|
+
# Adds +delay+, +delay_for+ and +delay_until+ methods to ActionMailer to offload arbitrary email
|
9
|
+
# delivery to Sidekiq.
|
10
10
|
#
|
11
|
+
# @example
|
11
12
|
# UserMailer.delay.send_welcome_email(new_user)
|
12
13
|
# UserMailer.delay_for(5.days).send_welcome_email(new_user)
|
13
14
|
# UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
|
@@ -5,7 +5,7 @@ require "sidekiq/extensions/generic_proxy"
|
|
5
5
|
module Sidekiq
|
6
6
|
module Extensions
|
7
7
|
##
|
8
|
-
# Adds
|
8
|
+
# Adds +delay+, +delay_for+ and +delay_until+ methods to ActiveRecord to offload instance method
|
9
9
|
# execution to Sidekiq.
|
10
10
|
#
|
11
11
|
# @example
|
@@ -10,7 +10,7 @@ module Sidekiq
|
|
10
10
|
def initialize(performable, target, options = {})
|
11
11
|
@performable = performable
|
12
12
|
@target = target
|
13
|
-
@opts = options
|
13
|
+
@opts = options.transform_keys(&:to_s)
|
14
14
|
end
|
15
15
|
|
16
16
|
def method_missing(name, *args)
|
@@ -24,7 +24,9 @@ module Sidekiq
|
|
24
24
|
if marshalled.size > SIZE_LIMIT
|
25
25
|
::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
|
26
26
|
end
|
27
|
-
@performable.client_push({"class" => @performable,
|
27
|
+
@performable.client_push({"class" => @performable,
|
28
|
+
"args" => [marshalled],
|
29
|
+
"display_class" => "#{@target}.#{name}"}.merge(@opts))
|
28
30
|
end
|
29
31
|
end
|
30
32
|
end
|